{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\conda\\envs\\evaluate_env\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "# step1 导入相关包\n",
    "import evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\conda\\envs\\evaluate_env\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "Couldn't find a module script at d:\\ghh\\llm_project\\evaluate\\exact_match\\exact_match.py. Module 'exact_match' doesn't exist on the Hugging Face Hub either.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 4\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[39m# 正确的写法\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[39mimport\u001b[39;00m\u001b[39m \u001b[39m\u001b[39mevaluate\u001b[39;00m\n\u001b[1;32m----> 4\u001b[0m exact_match \u001b[39m=\u001b[39m evaluate\u001b[39m.\u001b[39;49mload(\u001b[39m'\u001b[39;49m\u001b[39mexact_match\u001b[39;49m\u001b[39m'\u001b[39;49m)\n\u001b[0;32m      5\u001b[0m result \u001b[39m=\u001b[39m exact_match\u001b[39m.\u001b[39mcompute(references\u001b[39m=\u001b[39m[\u001b[39m'\u001b[39m\u001b[39mhello\u001b[39m\u001b[39m'\u001b[39m], predictions\u001b[39m=\u001b[39m[\u001b[39m'\u001b[39m\u001b[39mhello\u001b[39m\u001b[39m'\u001b[39m])\n\u001b[0;32m      6\u001b[0m \u001b[39mprint\u001b[39m(result)\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\evaluate\\loading.py:748\u001b[0m, in \u001b[0;36mload\u001b[1;34m(path, config_name, module_type, process_id, num_process, cache_dir, experiment_id, keep_in_memory, download_config, download_mode, revision, **init_kwargs)\u001b[0m\n\u001b[0;32m    703\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Load a [`~evaluate.EvaluationModule`].\u001b[39;00m\n\u001b[0;32m    704\u001b[0m \n\u001b[0;32m    705\u001b[0m \u001b[39mArgs:\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    745\u001b[0m \u001b[39m    ```\u001b[39;00m\n\u001b[0;32m    746\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m    747\u001b[0m download_mode \u001b[39m=\u001b[39m DownloadMode(download_mode \u001b[39mor\u001b[39;00m DownloadMode\u001b[39m.\u001b[39mREUSE_DATASET_IF_EXISTS)\n\u001b[1;32m--> 748\u001b[0m evaluation_module \u001b[39m=\u001b[39m evaluation_module_factory(\n\u001b[0;32m    749\u001b[0m     path, module_type\u001b[39m=\u001b[39;49mmodule_type, revision\u001b[39m=\u001b[39;49mrevision, download_config\u001b[39m=\u001b[39;49mdownload_config, download_mode\u001b[39m=\u001b[39;49mdownload_mode\n\u001b[0;32m    750\u001b[0m )\n\u001b[0;32m    751\u001b[0m evaluation_cls \u001b[39m=\u001b[39m import_main_class(evaluation_module\u001b[39m.\u001b[39mmodule_path)\n\u001b[0;32m    752\u001b[0m evaluation_instance \u001b[39m=\u001b[39m evaluation_cls(\n\u001b[0;32m    753\u001b[0m     config_name\u001b[39m=\u001b[39mconfig_name,\n\u001b[0;32m    754\u001b[0m     process_id\u001b[39m=\u001b[39mprocess_id,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    760\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39minit_kwargs,\n\u001b[0;32m    761\u001b[0m )\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\evaluate\\loading.py:681\u001b[0m, in \u001b[0;36mevaluation_module_factory\u001b[1;34m(path, module_type, revision, download_config, download_mode, force_local_path, dynamic_modules_path, **download_kwargs)\u001b[0m\n\u001b[0;32m    679\u001b[0m         \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(e1, (\u001b[39mConnectionError\u001b[39;00m, \u001b[39mFileNotFoundError\u001b[39;00m)):\n\u001b[0;32m    680\u001b[0m             \u001b[39mraise\u001b[39;00m e1 \u001b[39mfrom\u001b[39;00m\u001b[39m \u001b[39m\u001b[39mNone\u001b[39;00m\n\u001b[1;32m--> 681\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mFileNotFoundError\u001b[39;00m(\n\u001b[0;32m    682\u001b[0m             \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCouldn\u001b[39m\u001b[39m'\u001b[39m\u001b[39mt find a module script at \u001b[39m\u001b[39m{\u001b[39;00mrelative_to_absolute_path(combined_path)\u001b[39m}\u001b[39;00m\u001b[39m. \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    683\u001b[0m             \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mModule \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mpath\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m doesn\u001b[39m\u001b[39m'\u001b[39m\u001b[39mt exist on the Hugging Face Hub either.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    684\u001b[0m         ) \u001b[39mfrom\u001b[39;00m\u001b[39m \u001b[39m\u001b[39mNone\u001b[39;00m\n\u001b[0;32m    685\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m    686\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mFileNotFoundError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCouldn\u001b[39m\u001b[39m'\u001b[39m\u001b[39mt find a module script at \u001b[39m\u001b[39m{\u001b[39;00mrelative_to_absolute_path(combined_path)\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: Couldn't find a module script at d:\\ghh\\llm_project\\evaluate\\exact_match\\exact_match.py. Module 'exact_match' doesn't exist on the Hugging Face Hub either."
     ]
    }
   ],
   "source": [
    "# 正确的写法\n",
    "import evaluate\n",
    "\n",
    "exact_match = evaluate.load('exact_match')\n",
    "result = exact_match.compute(references=['hello'], predictions=['hello'])\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "可用的评估指标：\n"
     ]
    },
    {
     "ename": "ConnectTimeout",
     "evalue": "HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /api/spaces?filter=metric (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000002FB8B2D6470>, 'Connection to huggingface.co timed out. (connect timeout=None)'))",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTimeoutError\u001b[0m                              Traceback (most recent call last)",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\connection.py:198\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    197\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 198\u001b[0m     sock \u001b[39m=\u001b[39m connection\u001b[39m.\u001b[39;49mcreate_connection(\n\u001b[0;32m    199\u001b[0m         (\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_dns_host, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mport),\n\u001b[0;32m    200\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtimeout,\n\u001b[0;32m    201\u001b[0m         source_address\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msource_address,\n\u001b[0;32m    202\u001b[0m         socket_options\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msocket_options,\n\u001b[0;32m    203\u001b[0m     )\n\u001b[0;32m    204\u001b[0m \u001b[39mexcept\u001b[39;00m socket\u001b[39m.\u001b[39mgaierror \u001b[39mas\u001b[39;00m e:\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\util\\connection.py:85\u001b[0m, in \u001b[0;36mcreate_connection\u001b[1;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[0;32m     84\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> 85\u001b[0m     \u001b[39mraise\u001b[39;00m err\n\u001b[0;32m     86\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[0;32m     87\u001b[0m     \u001b[39m# Break explicitly a reference cycle\u001b[39;00m\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\util\\connection.py:73\u001b[0m, in \u001b[0;36mcreate_connection\u001b[1;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[0;32m     72\u001b[0m     sock\u001b[39m.\u001b[39mbind(source_address)\n\u001b[1;32m---> 73\u001b[0m sock\u001b[39m.\u001b[39;49mconnect(sa)\n\u001b[0;32m     74\u001b[0m \u001b[39m# Break explicitly a reference cycle\u001b[39;00m\n",
      "\u001b[1;31mTimeoutError\u001b[0m: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mConnectTimeoutError\u001b[0m                       Traceback (most recent call last)",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\connectionpool.py:787\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[0;32m    786\u001b[0m \u001b[39m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[1;32m--> 787\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_make_request(\n\u001b[0;32m    788\u001b[0m     conn,\n\u001b[0;32m    789\u001b[0m     method,\n\u001b[0;32m    790\u001b[0m     url,\n\u001b[0;32m    791\u001b[0m     timeout\u001b[39m=\u001b[39mtimeout_obj,\n\u001b[0;32m    792\u001b[0m     body\u001b[39m=\u001b[39mbody,\n\u001b[0;32m    793\u001b[0m     headers\u001b[39m=\u001b[39mheaders,\n\u001b[0;32m    794\u001b[0m     chunked\u001b[39m=\u001b[39mchunked,\n\u001b[0;32m    795\u001b[0m     retries\u001b[39m=\u001b[39mretries,\n\u001b[0;32m    796\u001b[0m     response_conn\u001b[39m=\u001b[39mresponse_conn,\n\u001b[0;32m    797\u001b[0m     preload_content\u001b[39m=\u001b[39mpreload_content,\n\u001b[0;32m    798\u001b[0m     decode_content\u001b[39m=\u001b[39mdecode_content,\n\u001b[0;32m    799\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mresponse_kw,\n\u001b[0;32m    800\u001b[0m )\n\u001b[0;32m    802\u001b[0m \u001b[39m# Everything went great!\u001b[39;00m\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\connectionpool.py:488\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[1;34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[0;32m    487\u001b[0m         new_e \u001b[39m=\u001b[39m _wrap_proxy_error(new_e, conn\u001b[39m.\u001b[39mproxy\u001b[39m.\u001b[39mscheme)\n\u001b[1;32m--> 488\u001b[0m     \u001b[39mraise\u001b[39;00m new_e\n\u001b[0;32m    490\u001b[0m \u001b[39m# conn.request() calls http.client.*.request, not the method in\u001b[39;00m\n\u001b[0;32m    491\u001b[0m \u001b[39m# urllib3.request. It also calls makefile (recv) on the socket.\u001b[39;00m\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\connectionpool.py:464\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[1;34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[0;32m    463\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 464\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_validate_conn(conn)\n\u001b[0;32m    465\u001b[0m \u001b[39mexcept\u001b[39;00m (SocketTimeout, BaseSSLError) \u001b[39mas\u001b[39;00m e:\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\connectionpool.py:1093\u001b[0m, in \u001b[0;36mHTTPSConnectionPool._validate_conn\u001b[1;34m(self, conn)\u001b[0m\n\u001b[0;32m   1092\u001b[0m \u001b[39mif\u001b[39;00m conn\u001b[39m.\u001b[39mis_closed:\n\u001b[1;32m-> 1093\u001b[0m     conn\u001b[39m.\u001b[39;49mconnect()\n\u001b[0;32m   1095\u001b[0m \u001b[39m# TODO revise this, see https://github.com/urllib3/urllib3/issues/2791\u001b[39;00m\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\connection.py:753\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    752\u001b[0m sock: socket\u001b[39m.\u001b[39msocket \u001b[39m|\u001b[39m ssl\u001b[39m.\u001b[39mSSLSocket\n\u001b[1;32m--> 753\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msock \u001b[39m=\u001b[39m sock \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_new_conn()\n\u001b[0;32m    754\u001b[0m server_hostname: \u001b[39mstr\u001b[39m \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhost\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\connection.py:207\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    206\u001b[0m \u001b[39mexcept\u001b[39;00m SocketTimeout \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m--> 207\u001b[0m     \u001b[39mraise\u001b[39;00m ConnectTimeoutError(\n\u001b[0;32m    208\u001b[0m         \u001b[39mself\u001b[39m,\n\u001b[0;32m    209\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mConnection to \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhost\u001b[39m}\u001b[39;00m\u001b[39m timed out. (connect timeout=\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtimeout\u001b[39m}\u001b[39;00m\u001b[39m)\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m    210\u001b[0m     ) \u001b[39mfrom\u001b[39;00m\u001b[39m \u001b[39m\u001b[39me\u001b[39;00m\n\u001b[0;32m    212\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mOSError\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n",
      "\u001b[1;31mConnectTimeoutError\u001b[0m: (<urllib3.connection.HTTPSConnection object at 0x000002FB8B2D6470>, 'Connection to huggingface.co timed out. (connect timeout=None)')",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mMaxRetryError\u001b[0m                             Traceback (most recent call last)",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\requests\\adapters.py:644\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[1;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[0;32m    643\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 644\u001b[0m     resp \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39;49murlopen(\n\u001b[0;32m    645\u001b[0m         method\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mmethod,\n\u001b[0;32m    646\u001b[0m         url\u001b[39m=\u001b[39;49murl,\n\u001b[0;32m    647\u001b[0m         body\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mbody,\n\u001b[0;32m    648\u001b[0m         headers\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mheaders,\n\u001b[0;32m    649\u001b[0m         redirect\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m    650\u001b[0m         assert_same_host\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m    651\u001b[0m         preload_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m    652\u001b[0m         decode_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m    653\u001b[0m         retries\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_retries,\n\u001b[0;32m    654\u001b[0m         timeout\u001b[39m=\u001b[39;49mtimeout,\n\u001b[0;32m    655\u001b[0m         chunked\u001b[39m=\u001b[39;49mchunked,\n\u001b[0;32m    656\u001b[0m     )\n\u001b[0;32m    658\u001b[0m \u001b[39mexcept\u001b[39;00m (ProtocolError, \u001b[39mOSError\u001b[39;00m) \u001b[39mas\u001b[39;00m err:\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\connectionpool.py:841\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[0;32m    839\u001b[0m     new_e \u001b[39m=\u001b[39m ProtocolError(\u001b[39m\"\u001b[39m\u001b[39mConnection aborted.\u001b[39m\u001b[39m\"\u001b[39m, new_e)\n\u001b[1;32m--> 841\u001b[0m retries \u001b[39m=\u001b[39m retries\u001b[39m.\u001b[39;49mincrement(\n\u001b[0;32m    842\u001b[0m     method, url, error\u001b[39m=\u001b[39;49mnew_e, _pool\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m, _stacktrace\u001b[39m=\u001b[39;49msys\u001b[39m.\u001b[39;49mexc_info()[\u001b[39m2\u001b[39;49m]\n\u001b[0;32m    843\u001b[0m )\n\u001b[0;32m    844\u001b[0m retries\u001b[39m.\u001b[39msleep()\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\urllib3\\util\\retry.py:519\u001b[0m, in \u001b[0;36mRetry.increment\u001b[1;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[0;32m    518\u001b[0m     reason \u001b[39m=\u001b[39m error \u001b[39mor\u001b[39;00m ResponseError(cause)\n\u001b[1;32m--> 519\u001b[0m     \u001b[39mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[39mfrom\u001b[39;00m\u001b[39m \u001b[39m\u001b[39mreason\u001b[39;00m  \u001b[39m# type: ignore[arg-type]\u001b[39;00m\n\u001b[0;32m    521\u001b[0m log\u001b[39m.\u001b[39mdebug(\u001b[39m\"\u001b[39m\u001b[39mIncremented Retry for (url=\u001b[39m\u001b[39m'\u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m): \u001b[39m\u001b[39m%r\u001b[39;00m\u001b[39m\"\u001b[39m, url, new_retry)\n",
      "\u001b[1;31mMaxRetryError\u001b[0m: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /api/spaces?filter=metric (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000002FB8B2D6470>, 'Connection to huggingface.co timed out. (connect timeout=None)'))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mConnectTimeout\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[2], line 3\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[39m# 查看所有可用的指标\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39m可用的评估指标：\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m----> 3\u001b[0m metrics \u001b[39m=\u001b[39m evaluate\u001b[39m.\u001b[39;49mlist_evaluation_modules()\n\u001b[0;32m      4\u001b[0m \u001b[39mprint\u001b[39m(metrics)\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\evaluate\\inspect.py:61\u001b[0m, in \u001b[0;36mlist_evaluation_modules\u001b[1;34m(module_type, include_community, with_details)\u001b[0m\n\u001b[0;32m     58\u001b[0m     evaluations_list \u001b[39m=\u001b[39m []\n\u001b[0;32m     59\u001b[0m     \u001b[39mfor\u001b[39;00m module_type \u001b[39min\u001b[39;00m EVALUATION_MODULE_TYPES:\n\u001b[0;32m     60\u001b[0m         evaluations_list\u001b[39m.\u001b[39mextend(\n\u001b[1;32m---> 61\u001b[0m             _list_evaluation_modules_type(\n\u001b[0;32m     62\u001b[0m                 module_type, include_community\u001b[39m=\u001b[39;49minclude_community, with_details\u001b[39m=\u001b[39;49mwith_details\n\u001b[0;32m     63\u001b[0m             )\n\u001b[0;32m     64\u001b[0m         )\n\u001b[0;32m     65\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m     66\u001b[0m     \u001b[39mif\u001b[39;00m module_type \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m EVALUATION_MODULE_TYPES:\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\evaluate\\inspect.py:76\u001b[0m, in \u001b[0;36m_list_evaluation_modules_type\u001b[1;34m(module_type, include_community, with_details)\u001b[0m\n\u001b[0;32m     74\u001b[0m \u001b[39mdef\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m_list_evaluation_modules_type\u001b[39m(module_type, include_community\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m, with_details\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m):\n\u001b[1;32m---> 76\u001b[0m     r \u001b[39m=\u001b[39m requests\u001b[39m.\u001b[39;49mget(HF_LIST_ENDPOINT\u001b[39m.\u001b[39;49mformat(\u001b[39mtype\u001b[39;49m\u001b[39m=\u001b[39;49mmodule_type))\n\u001b[0;32m     77\u001b[0m     r\u001b[39m.\u001b[39mraise_for_status()\n\u001b[0;32m     78\u001b[0m     d \u001b[39m=\u001b[39m r\u001b[39m.\u001b[39mjson()\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\requests\\api.py:73\u001b[0m, in \u001b[0;36mget\u001b[1;34m(url, params, **kwargs)\u001b[0m\n\u001b[0;32m     62\u001b[0m \u001b[39mdef\u001b[39;00m\u001b[39m \u001b[39m\u001b[39mget\u001b[39m(url, params\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[0;32m     63\u001b[0m \u001b[39m    \u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"Sends a GET request.\u001b[39;00m\n\u001b[0;32m     64\u001b[0m \n\u001b[0;32m     65\u001b[0m \u001b[39m    :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     70\u001b[0m \u001b[39m    :rtype: requests.Response\u001b[39;00m\n\u001b[0;32m     71\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[1;32m---> 73\u001b[0m     \u001b[39mreturn\u001b[39;00m request(\u001b[39m\"\u001b[39m\u001b[39mget\u001b[39m\u001b[39m\"\u001b[39m, url, params\u001b[39m=\u001b[39mparams, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\requests\\api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[1;34m(method, url, **kwargs)\u001b[0m\n\u001b[0;32m     55\u001b[0m \u001b[39m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[0;32m     56\u001b[0m \u001b[39m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[0;32m     57\u001b[0m \u001b[39m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[0;32m     58\u001b[0m \u001b[39mwith\u001b[39;00m sessions\u001b[39m.\u001b[39mSession() \u001b[39mas\u001b[39;00m session:\n\u001b[1;32m---> 59\u001b[0m     \u001b[39mreturn\u001b[39;00m session\u001b[39m.\u001b[39mrequest(method\u001b[39m=\u001b[39mmethod, url\u001b[39m=\u001b[39murl, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\requests\\sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[1;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[0;32m    584\u001b[0m send_kwargs \u001b[39m=\u001b[39m {\n\u001b[0;32m    585\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mtimeout\u001b[39m\u001b[39m\"\u001b[39m: timeout,\n\u001b[0;32m    586\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mallow_redirects\u001b[39m\u001b[39m\"\u001b[39m: allow_redirects,\n\u001b[0;32m    587\u001b[0m }\n\u001b[0;32m    588\u001b[0m send_kwargs\u001b[39m.\u001b[39mupdate(settings)\n\u001b[1;32m--> 589\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msend(prep, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39msend_kwargs)\n\u001b[0;32m    591\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\requests\\sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[1;34m(self, request, **kwargs)\u001b[0m\n\u001b[0;32m    700\u001b[0m start \u001b[39m=\u001b[39m preferred_clock()\n\u001b[0;32m    702\u001b[0m \u001b[39m# Send the request\u001b[39;00m\n\u001b[1;32m--> 703\u001b[0m r \u001b[39m=\u001b[39m adapter\u001b[39m.\u001b[39msend(request, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    705\u001b[0m \u001b[39m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[0;32m    706\u001b[0m elapsed \u001b[39m=\u001b[39m preferred_clock() \u001b[39m-\u001b[39m start\n",
      "File \u001b[1;32md:\\conda\\envs\\evaluate_env\\lib\\site-packages\\requests\\adapters.py:665\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[1;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[0;32m    662\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(e\u001b[39m.\u001b[39mreason, ConnectTimeoutError):\n\u001b[0;32m    663\u001b[0m     \u001b[39m# TODO: Remove this in 3.0.0: see #2811\u001b[39;00m\n\u001b[0;32m    664\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(e\u001b[39m.\u001b[39mreason, NewConnectionError):\n\u001b[1;32m--> 665\u001b[0m         \u001b[39mraise\u001b[39;00m ConnectTimeout(e, request\u001b[39m=\u001b[39mrequest)\n\u001b[0;32m    667\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(e\u001b[39m.\u001b[39mreason, ResponseError):\n\u001b[0;32m    668\u001b[0m     \u001b[39mraise\u001b[39;00m RetryError(e, request\u001b[39m=\u001b[39mrequest)\n",
      "\u001b[1;31mConnectTimeout\u001b[0m: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /api/spaces?filter=metric (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000002FB8B2D6470>, 'Connection to huggingface.co timed out. (connect timeout=None)'))"
     ]
    }
   ],
   "source": [
    "# 查看所有可用的指标\n",
    "print(\"可用的评估指标：\")\n",
    "metrics = evaluate.list_evaluation_modules()\n",
    "print(metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "错误: Couldn't find a module script at d:\\ghh\\llm_project\\evaluate\\accuracy\\accuracy.py. Module 'accuracy' doesn't exist on the Hugging Face Hub either.\n"
     ]
    }
   ],
   "source": [
    "import evaluate\n",
    "import os\n",
    "\n",
    "# 设置环境变量为离线模式\n",
    "os.environ[\"HF_DATASETS_OFFLINE\"] = \"1\"\n",
    "os.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\n",
    "\n",
    "# 尝试加载\n",
    "try:\n",
    "  metric = evaluate.load(\"accuracy\")\n",
    "  result = metric.compute(references=[0, 1], predictions=[0, 1])\n",
    "  print(result)\n",
    "except Exception as e:\n",
    "  print(f\"错误: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'exact_match': 1.0}\n",
      "{'exact_match': 0.5}\n"
     ]
    }
   ],
   "source": [
    "# 手动实现精确匹配指标\n",
    "def exact_match(references, predictions):\n",
    "  \"\"\"\n",
    "  计算精确匹配分数\n",
    "  \"\"\"\n",
    "  if len(references) != len(predictions):\n",
    "    raise ValueError(\"references 和 predictions 长度必须相同\")\n",
    "  \n",
    "  correct = 0\n",
    "  for ref, pred in zip(references, predictions):\n",
    "    if ref == pred:\n",
    "      correct += 1\n",
    "  \n",
    "  return {\"exact_match\": correct / len(references)}\n",
    "\n",
    "# 测试\n",
    "result = exact_match(references=['hello', 'world'], predictions=['hello', 'world'])\n",
    "print(result)  # 输出: {'exact_match': 1.0}\n",
    "\n",
    "result = exact_match(references=['hello', 'world'], predictions=['hello', 'test'])\n",
    "print(result)  # 输出: {'exact_match': 0.5}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "evaluate_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.18"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "37990cb7e14a4280d29cc47a919e2d55f3bfd74dc413cee212f628168b6511e1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
